# -*- coding: utf-8 -*-
# @Time    : 2019/2/26 17:59
# @Author  : yueconger
# @File    : redis_pool.py
# 维持访问队列
import json
import re
import sys
import redis


class RedisPool(object):
    def __init__(self):
        pass

    def question_set(self):
        pool = redis.ConnectionPool(host='127.0.0.1', port=6379)
        r = redis.Redis(connection_pool=pool)
        pipe = r.pipeline()
        pipe.set('one', 'first')
        pipe.set('two', 'second')
        pipe.execute()


class QuesQueue(object):
    '''问题队列'''

    def __init__(self, *, new=False, seed: list = None):
        '''如果新开始爬虫选择new=True，并传入种子问题列表
        :param seed: 初始待爬取问题列表
        '''
        self.redis = redis.StrictRedis(db=5)
        if new:
            if not seed:
                print('请传入种子问题集！')
                sys.exit(11)
            self.redis.flushdb()
            self.redis.sadd('question', *seed)

    def pop(self):
        '''pop random one from question set and add it to crawled set'''
        pop = self.redis.spop('question')
        if pop:
            self.redis.sadd('question_crawled', pop)
            return pop.decode('utf8')
        else:
            return False

    def add(self, *values):
        '''将问题加入待爬取队列
        仅当该问题不在已爬取set中时加入'''
        for value in values:
            value = value.strip()
            if not value:
                continue
            if not self.redis.sismember('question_crawled', value):
                self.redis.sadd('question', value)


if __name__ == '__main__':
    with open('qa_pool.txt', 'r', encoding='utf-8') as f:
        con_list = f.readlines()
    qa_list = []
    for con in con_list:
        question = con.strip()
        qa_list.append(question)
    queue = QuesQueue(new=True, seed=qa_list)

    qa = 'kkkkk'
    queue.add(*qa)
